
runJobPostingProcess <- function() {
  library(XML)
  library(stringr)
  rootOutput <- 'C:/SalesInsight/JobSearch/Postings/'
  rootControl <- 'C:/SalesInsight/JobSearch/ControlFiles/'
  algoFile <- 'C:/SalesInsight/JobSearch/algoInfo.txt'
  
  compProdKey <- read.table(file=str_c(algoFile), quote='',header=FALSE, sep=',', col.names=c('algo','algoId'), colClasses=c('character','character') )
  
  for(p in 1:nrow(compProdKey)) {
    algo <- compProdKey$algo[p]
    algoId <- compProdKey$algoId[p]
    
    rootOutput<- str_c(rootOutput,algoId)
    if(!file.exists(rootOutput)) {
      dir.create(rootOutput)
    }
    rootOutput<- str_c(rootOutput,'/')
    
    rootControl <- str_c(rootControl,algoId)
    if(!file.exists(rootControl)) {
      dir.create(rootControl)
    }
    rootControl <- str_c(rootControl, '/')
    
    downloadJobPostings(algo, algoId, rootOutput, rootControl)
  }
}

downloadJobPostings <- function(algo, algoId,  rootOutput, rootControl) {
  logInfo(rootControl, algoId, str_c('Started processing postings for -> ',algoId))
  logInfo(rootControl, algoId, str_c('Algo -> ',algo))
  tryCatch( {
    companyData <- data.frame(product=NULL,company=NULL,jobkey=NULL,location=NULL,postDate=NULL)
    countUrl <- str_c('http://api.indeed.com/ads/apisearch?v=2&publisher=2487201386265292&',
                      'q=',algo,'&start=')
    countXML <- xmlParse(countUrl)
    countDF <- xmlToDataFrame(getNodeSet(countXML, "//totalresults"),stringsAsFactors=FALSE)
    count <- as.numeric(countDF$text)
    logInfo(rootControl, algoId, str_c('Number of postings found for -> ',algoId ,'=', count))
    stringDate <- format(Sys.time(), "%b_%d_%Y")
    fileName <- str_c(rootOutput, algoId, '_', stringDate, '.csv')
    if(file.exists(fileName)) {
      logInfo(rootControl, algoId, str_c('Data file exists..deleting todays file to recreate, fileName -> ', fileName))
      file.remove(fileName)
    }
    start <- 0  
    for(i in 0: floor(count/25)) {
     tryCatch( { 
         start <- i * 25 + 1 
         url <- str_c('http://api.indeed.com/ads/apisearch?publisher=2487201386265292&',
                      'q=',algo,'&limit=25&start=',start,'&v=2')
         
         logInfo(rootControl, algoId, str_c('Now Processing Link -> ', url))
         results <- xmlParse(url)
         data <- xmlToDataFrame(getNodeSet(results, "//result"),stringsAsFactors=FALSE)   
         companyData <- rbind(companyData,data.frame(product=algoId,
                                                     company=data$company,data$jobkey,
                                                     location=data$formattedLocation,
                                                     date=data$date))
         }, error = function(e) {
           logInfo(rootControl, algoId, str_c('error getting jobs in URL -> ', url))
         })
    }
    write.table(col.names=FALSE, quote=FALSE, companyData ,file=fileName,append=TRUE, sep='|' ,eol='\r\n', row.names=FALSE)
  }, error = function(e) {
    logInfo(rootControl, algoId, str_c('error getting count for algo id -> ', algoId))
  })
  logInfo(rootControl, algoId, str_c('Done processing postings for -> ',algoId)) 
}


logInfo <- function(rootControl, algoId, message='no message passed') {
  infoFile <- str_c(rootControl,'_', algoId, '_', stringDate,'_info.txt')
  write(message, file=infoFile, append=TRUE, ncolumns=1)
}
